from bs4 import BeautifulSoup
import requests
import time

# 爬去北京地区短租房信息
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"}


def judgment_sex(class_name):
    if class_name == ['member_icol']:
        return '女'
    else:
        return '男'


def get_links(url):
    wb_data = requests.get(url, headers=headers)
    soup = BeautifulSoup(wb_data.text, "html.parser")  # TODO lxml不管用
    links = soup.select('#page_list > ul > li > a')
    for link in links:
        href = link.get("href")
        # print(href)
        get_info(href)


def get_info(url):
    wb_data = requests.get(url, headers=headers)
    soup = BeautifulSoup(wb_data.text, 'html.parser')
    # print(soup.text)
    tittles = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
    addresses = soup.select("body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span")
    prices = soup.select("#pricePart > div.day_l > span")
    imgs = soup.select('#imgMouseCusor')
    names = soup.select("#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a")
    sexs = soup.select("#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span")
    for tittle, address, price, img, name, sex in zip(tittles, addresses, prices, imgs, names, sexs):
        data = {'title': tittle.get_text().strip(),
                'address': address.get_text().strip(),
                'price': price.get_text(),
                'img': img.get('src'),
                'name': name.get_text(),
                'sex': judgment_sex(sex.get('class'))}
        print(data)


if __name__ == '__main__':
    urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1, 14)]
    for single_url in urls:
        # print(single_url)
        get_links(single_url)
        time.sleep(2)
